###############################################################
## check job stauts ##
## usage: ./check_submitted_jobs.sh $1
##        arg1: history file (contains jobID, output, Njobs) 
## Lailin, Ann Arbor, 1.2013
###############################################################


Retry_dir=Retry
if [ ! -d $Retry_dir ]; then mkdir $Retry_dir; fi

timetag=`date +%Y%m%d%H%M`
check_jobID=-1
nFinished=-1
checkFailedID() {
    pbook -c "show($1)"  > tmp_status
    less tmp_status >> out_$timetag
    jobStatus=tmp_status
    nRunning=`less $jobStatus | grep running`
    nFailed=`less $jobStatus | grep failed | awk '{print $3}'`
    nCancelled=`less $jobStatus | grep cancelled | awk '{print $3}'`
    nFinished=`less $jobStatus | grep finished | awk '{print $3}'`
    if [ -z "${nFinished}" ]; then nFinished=0; fi
    echo "failed: $nFailed; cancelled: $nCancelled; finished: $nFinished"
    if [ -n "${nRunning}" ]; then
        # running 
        check_jobID=-2   
    elif [ -n "${nFailed}" ] || [ -n "${nCancelled}" ]; then
        retrySetID=`less $jobStatus | grep retrySetID | awk '{print $3}'`
        retryID=`less $jobStatus | grep retryID | awk '{print $3}'`
        if [ -n "${retryID}" ] && [ "${retryID}" != "0" ]; then
            checkFailedID $retryID
        elif [ -n "${retrySetID}" ] && [ "${retrySetID}" != "0" ]; then
            checkFailedID $retrySetID
        else 
            # failed
            check_jobID=$1 
        fi
    else 
        # succeed   
        check_jobID=-1 
    fi

}

pbook -c "sync()"

while read job output Njobs; do
    if [ -z "${job}" ] || [ -z "${output}" ] || [ -z "${Njobs}" ]; then
        continue
    fi
    if [ "Njobs" == "FAIL" ]; then
        echo "Warning => Failed jobs: $output"
        continue
    fi
    check_jobID=$job
    nFinished=-1
    checkFailedID $job
    echo "failed_jobID: $check_jobID"
    if [ "${check_jobID}" == "-2" ]; then
        echo "${job} ${output} ${Njobs}" >> $Retry_dir/running_job_$timetag
    elif [ "${check_jobID}" == "-1" ]; then
        echo "${job} ${output} ${Njobs}" >> $Retry_dir/completed_job_$timetag
    else 
        echo "${job} ${output} ${Njobs}" >> $Retry_dir/failed_job_$timetag
        echo "${output}" >> $Retry_dir/failed_run_$timetag
    fi
#done < history/submitted
done < history/20130108_failed_run_v4_log.txt
